Teach fsck about partial commits

An OSTree user noticed that `ostree fsck` would produce `missing
object` errors in the case of interrupted pulls.

It's possible to do e.g. `ostree pull --subpath=/usr/share/rpm ...`,
which gets you just that portion of the commit.  The use case for this
was being able to see what changes would appear in an update before
actually downloading all of it.

(I think this would be better covered by static deltas, but those
 aren't final yet, and `--subpath` predates it)

Further, `.commitpartial` is used as a successor to the `transaction`
symlink for more precise knowledge in the case where a pull was
interrupted that we needed to resume scanning.

So it makes sense for `ostree fsck` to be aware of it.

diff --git a/Makefile-tests.am b/Makefile-tests.am
index bf584240dfc85aa1f5cee1d4bdfece01ff7d95d8..7350032f114fbe2fab0a0e83bd8666ffcfb78fe1 100644 (file)
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -48,6 +48,7 @@ testfiles = test-basic \
        test-admin-locking \
        test-repo-checkout-subpath      \
        test-reset-nonlinear \
+       test-oldstyle-partial \
        test-setuid \
        test-delta \
        test-xattrs \

diff --git a/doc/ostree-sections.txt b/doc/ostree-sections.txt
index 9e2ef6c35eee741607c04bc4b22781b1623dba66..b75efc97b410966912bef9f2bb0dd97bcceb1065 100644 (file)
--- a/doc/ostree-sections.txt
+++ b/doc/ostree-sections.txt
@@ -255,6 +255,7 @@ ostree_repo_write_content_finish
 ostree_repo_resolve_rev
 ostree_repo_list_refs
 ostree_repo_load_variant
+ostree_repo_load_commit
 ostree_repo_load_variant_if_exists
 ostree_repo_load_file
 ostree_repo_load_object_stream

diff --git a/src/libostree/ostree-cmdprivate.c b/src/libostree/ostree-cmdprivate.c
index ea12243485a19f216c097c81f3eabc1d45da822b..998349375a0ce3eaa9616cc097b8dd4245572c5d 100644 (file)
--- a/src/libostree/ostree-cmdprivate.c
+++ b/src/libostree/ostree-cmdprivate.c
@@ -21,6 +21,8 @@
 #include "config.h"
 
 #include "ostree-cmdprivate.h"
+#include "ostree-repo-private.h"
+#include "ostree-core-private.h"
 #include "ostree-sysroot.h"
 #include "ostree-bootloader-grub2.h"
 

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 97c044f88a3d9d6d7feec37687bc9e1df4dcfb1a..db91d1ecf4123e0a524f19eaff0c969168de8a55 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -1126,10 +1126,12 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
       /* For commits, check whether we only had a partial fetch */
       if (!do_scan && objtype == OSTREE_OBJECT_TYPE_COMMIT)
         {
-          g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (tmp_checksum);
-          struct stat stbuf;
+          OstreeRepoCommitState commitstate;
 
-          if (fstatat (pull_data->repo->repo_dir_fd, commitpartial_path, &stbuf, 0) == 0)
+          if (!ostree_repo_load_commit (pull_data->repo, tmp_checksum, NULL, &commitstate, error))
+            goto out;
+
+          if (commitstate & OSTREE_REPO_COMMIT_STATE_PARTIAL)
             {
               do_scan = TRUE;
               pull_data->commitpartial_exists = TRUE;
@@ -2178,14 +2180,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
           const char *checksum = value;
           g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
 
-          if (unlinkat (pull_data->repo->repo_dir_fd, commitpartial_path, 0) != 0)
-            {
-              if (errno != ENOENT)
-                {
-                  glnx_set_error_from_errno (error);
-                  goto out;
-                }
-            }
+          if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0))
+            goto out;
         }
         g_hash_table_iter_init (&hash_iter, commits_to_fetch);
         while (g_hash_table_iter_next (&hash_iter, &key, &value))
@@ -2193,14 +2189,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
             const char *commit = value;
             g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (commit);
 
-            if (unlinkat (pull_data->repo->repo_dir_fd, commitpartial_path, 0) != 0)
-              {
-                if (errno != ENOENT)
-                  {
-                    glnx_set_error_from_errno (error);
-                    goto out;
-                  }
-              }
+            if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0))
+              goto out;
           }
     }
 

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 5499dc1190d9bdb5010371723ace11100c73dd08..106298a4f8793bf304ba04b87d1a3524404df5b6 100644 (file)
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -2651,6 +2651,58 @@ ostree_repo_load_variant (OstreeRepo       *self,
                                  out_variant, NULL, NULL, NULL, error);
 }
 
+/**
+ * ostree_repo_load_commit:
+ * @self: Repo
+ * @checksum: Commit checksum
+ * @out_commit: (out) (allow-none): Commit
+ * @out_state: (out) (allow-none): Commit state
+ * @error: Error
+ *
+ * A version of ostree_repo_load_variant() specialized to commits,
+ * capable of returning extended state information.  Currently
+ * the only extended state is %OSTREE_REPO_COMMIT_STATE_PARTIAL, which
+ * means that only a sub-path of the commit is available.
+ */
+gboolean
+ostree_repo_load_commit (OstreeRepo            *self,
+                         const char            *checksum, 
+                         GVariant             **out_variant,
+                         OstreeRepoCommitState *out_state,
+                         GError               **error)
+{
+  gboolean ret = FALSE;
+
+  if (out_variant)
+    {
+      if (!load_metadata_internal (self, OSTREE_OBJECT_TYPE_COMMIT, checksum, TRUE,
+                                   out_variant, NULL, NULL, NULL, error))
+        goto out;
+    }
+
+  if (out_state)
+    {
+      g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
+      struct stat stbuf;
+
+      *out_state = 0;
+
+      if (fstatat (self->repo_dir_fd, commitpartial_path, &stbuf, 0) == 0)
+        {
+          *out_state |= OSTREE_REPO_COMMIT_STATE_PARTIAL;
+        }
+      else if (errno != ENOENT)
+        {
+          glnx_set_error_from_errno (error);
+          goto out;
+        }
+    }
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
 /**
  * ostree_repo_list_objects:
  * @self: Repo

diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index e6614dfa96c6c97899e6fdfb291623b5208dde08..c882356a05a076a5d34c371e5da9a48396d8f123 100644 (file)
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -280,6 +280,16 @@ gboolean      ostree_repo_load_variant_if_exists (OstreeRepo  *self,
                                                   GVariant     **out_variant,
                                                   GError       **error);
 
+typedef enum {
+  OSTREE_REPO_COMMIT_STATE_PARTIAL = (1 << 0),
+} OstreeRepoCommitState;
+
+gboolean      ostree_repo_load_commit (OstreeRepo            *self,
+                                       const char            *checksum, 
+                                       GVariant             **out_commit,
+                                       OstreeRepoCommitState *out_state,
+                                       GError               **error);
+
 gboolean ostree_repo_load_file (OstreeRepo         *self,
                                 const char         *checksum,
                                 GInputStream      **out_input,

diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c
index 040636bd57167d2966b196a85892472016d6ee72..b0f41a64bcc9a0a1b3afeded7c59372b53eebeba 100644 (file)
--- a/src/libotutil/ot-fs-utils.c
+++ b/src/libotutil/ot-fs-utils.c
@@ -22,6 +22,7 @@
 
 #include "ot-fs-utils.h"
 #include "libgsystem.h"
+#include "libglnx.h"
 #include <sys/xattr.h>
 #include <gio/gunixinputstream.h>
 
@@ -188,3 +189,20 @@ ot_openat_read_stream (int             dfd,
  out:
   return ret;
 }
+
+gboolean
+ot_ensure_unlinked_at (int dfd,
+                       const char *path,
+                       GError **error)
+{
+  if (unlinkat (dfd, path, 0) != 0)
+    {
+      if (G_UNLIKELY (errno != ENOENT))
+        {
+          glnx_set_error_from_errno (error);
+          return FALSE;
+        }
+    }
+  return TRUE;
+}
+

diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h
index 0113da4653dae29502f4d0c18ef1b3a9926b8cd4..10686be68e1c22f92d5896b485ab6a07fd7717df 100644 (file)
--- a/src/libotutil/ot-fs-utils.h
+++ b/src/libotutil/ot-fs-utils.h
@@ -57,4 +57,8 @@ gboolean ot_openat_read_stream (int             dfd,
                                 GCancellable   *cancellable,
                                 GError        **error);
 
+gboolean ot_ensure_unlinked_at (int dfd,
+                                const char *path,
+                                GError **error);
+
 G_END_DECLS

diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c
index ae1dcce4a0a2798a95e104937b619b727a2b16ff..8f340439377229376e02137ffabe67fe2d8f2d02 100644 (file)
--- a/src/ostree/ot-builtin-fsck.c
+++ b/src/ostree/ot-builtin-fsck.c
@@ -25,6 +25,7 @@
 #include "ot-main.h"
 #include "ot-builtins.h"
 #include "ostree.h"
+#include "ostree-cmdprivate.h"
 #include "otutil.h"
 
 static gboolean opt_quiet;
@@ -242,6 +243,7 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError **
   GHashTableIter hash_iter;
   gpointer key, value;
   gboolean found_corruption = FALSE;
+  guint n_partial = 0;
   gs_unref_hashtable GHashTable *objects = NULL;
   gs_unref_hashtable GHashTable *commits = NULL;
 
@@ -267,11 +269,22 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError **
       GVariant *serialized_key = key;
       const char *checksum;
       OstreeObjectType objtype;
+      OstreeRepoCommitState commitstate = 0;
 
       ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
 
       if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
-        g_hash_table_insert (commits, g_variant_ref (serialized_key), serialized_key);
+        {
+          if (!ostree_repo_load_commit (repo, checksum, NULL, &commitstate, error))
+            goto out;
+
+          if (commitstate & OSTREE_REPO_COMMIT_STATE_PARTIAL)
+            {
+              n_partial++;
+            }
+          else
+            g_hash_table_insert (commits, g_variant_ref (serialized_key), serialized_key);
+        }
     }
 
   g_clear_pointer (&objects, (GDestroyNotify) g_hash_table_unref);
@@ -284,6 +297,11 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError **
                                             cancellable, error))
     goto out;
 
+  if (n_partial > 0)
+    {
+      g_print ("%u partial commits not verified\n", n_partial);
+    }
+
   if (found_corruption)
     {
       g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,

diff --git a/tests/test-oldstyle-partial.sh b/tests/test-oldstyle-partial.sh
new file mode 100644 (file)
index 0000000..84460e9
--- /dev/null
+++ b/tests/test-oldstyle-partial.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive-z2"
+
+echo '1..1'
+
+cd ${test_tmpdir}
+rm repo -rf
+mkdir repo
+${CMD_PREFIX} ostree --repo=repo init
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+
+ostree --repo=repo pull origin main --subpath /baz
+ostree fsck --repo=repo >fsck.out
+assert_file_has_content fsck.out 'Verifying content integrity of 0 commit objects'
+assert_file_has_content fsck.out '1 partial commits not verified'